/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package getrootbahasa;

/**
 *
 * @author WaOnEmperoR
 */
import com.mysql.jdbc.PreparedStatement;
import com.stemaini.core.doc.Dokumen;
import com.stemaini.core.doc.Terms;
import com.stemaini.core.doc.Vektor;
import com.stemaini.core.database.Kamus;
import com.stemaini.core.doc.VektorNode;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import taini.koneksi.KoneksiAini;
import weka.classifiers.bayes.NaiveBayes;
import weka.classifiers.misc.SerializedClassifier;
import weka.core.Debug;
import weka.core.Instances;
import weka.core.converters.ArffLoader;

public class MainV2_TFIDF_Training {

    public static void main(String[] args){
        String isi = "";
        String kat ="";
        Iterator itr;
        int counter=0;

        KoneksiAini theKoneksi = new KoneksiAini();

        try {
            theKoneksi.connectFirst();
            String sql = "Select Isi, Kategori, Nomor_Dokumen from Berita";
            ResultSet res = theKoneksi.executeSelect(sql);
            while (res.next()){
                isi=(res.getString(1));
                kat=(res.getString(2));

                System.out.println(++counter);

                GetWords(isi, Integer.parseInt(res.getString(3)));

            }
        } catch (SQLException ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        }

        DefineIDF(270);
        UpdateTFIDF();
    }

    public static void DefineIDF(int jum_dok){//Jum_Dok = parameter N. Diisi sebanyak jumlah berita
        KoneksiAini theKoneksi = new KoneksiAini();
        String isi = "";
        String sql_in = "";
        String sql_in_2 = "";

        try {
            theKoneksi.connectFirst();
            String sql = "Select distinct Kata from Word_Frec_No";
            ResultSet res = theKoneksi.executeSelect(sql);
            while (res.next()){
                isi=(res.getString(1));

                sql_in = "select count(Nomor_Dokumen) from word_frec_no where Kata = '"+isi+"';";
                //System.out.println(sql_in);

                ResultSet res2 = theKoneksi.executeSelect(sql_in);

                while (res2.next()){
                    int jumlah = Integer.parseInt(res2.getString(1));
                    double idf = Math.log10(jum_dok/jumlah);

                    //Generate IDF
                    sql_in_2 = "INSERT INTO `word_idf` (`Word`, `Doc_Frec`, `Idf`) VALUES ('" + isi + "', " + jumlah + ", " + idf + ")";
                    System.out.println(sql_in_2);
                    theKoneksi.executeUpdate(sql_in_2);
                }

            }
        } catch (SQLException ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public static void UpdateTFIDF(){
        String kata="";
        double idf;

        KoneksiAini theKoneksi = new KoneksiAini();
        try {
            theKoneksi.connectFirst();
            String sql = "Select Word, IDF from Word_IDF";
            ResultSet res = theKoneksi.executeSelect(sql);
            while (res.next()){
               kata = res.getString(1);
               idf = Double.parseDouble(res.getString(2));

               String sql2 = "Update word_frec_no set TF_IDF = Berat * " + idf + "where Kata = '" + kata + "'";
               System.out.println(sql2);
               theKoneksi.executeUpdate(sql2);

            }
        } catch (SQLException ex) {
            Logger.getLogger(MainV2_TFIDF_Training.class.getName()).log(Level.SEVERE, null, ex);
        }

    }


    public static void GetWords(String str, int no_doc){
        KoneksiAini kon = new KoneksiAini();

        String content = str;
        String[] contentToken = content.split(" ");

        String sql = "";
        String kata = "";
        int frekuensi;
        double berat;

        String title = contentToken[0];
        String text = contentToken[0];

        Vektor vektor = preprocessing(content);

        for(int i = 0; i < vektor.getSize(); i++){
            VektorNode vn = vektor.getTermNode(i);
            //hasil+=(vn.getWord()+ "->" + vn.getFreq() + " ");
            kata = vn.getWord();
            frekuensi = vn.getFreq();
            berat = 1 + Math.log10(frekuensi);//Rumus untuk Wt,d

            //Hanya mencari Wt,d yang frekuensinya lebih dari 2
            if (frekuensi>2){
                try {
                    kon.connectFirst();
                    //Generate Term Frequency
                    sql = "INSERT INTO `word_frec_no` (`Nomor_Dokumen`, `Kata`, `Frekuensi`, `Berat`) VALUES (" + no_doc + ", '" + kata + "', " + frekuensi + ", " + berat + ")";
                    System.out.println(sql);
                    kon.executeUpdate(sql);
                } catch (SQLException ex) {
                    Logger.getLogger(MainV2_TFIDF_Training.class.getName()).log(Level.SEVERE, null, ex);
                }
            }
        }
    }

    public static Vektor preprocessing(String text) {
        try {
            KoneksiAini theKoneksi = new KoneksiAini();
            int pilihanStemmer = 3;
            theKoneksi.connectFirst();
            Dokumen dok;
            Terms terms = new Terms(theKoneksi);
            Kamus kamus = new Kamus(theKoneksi);
            kamus.initialize();
            dok = new Dokumen("1", text, kamus, terms, true);
            dok.doLowerCaseIsi();
            dok.doFiltering();
            dok.doTermExtraction();
            dok.doStemming(pilihanStemmer);
            return dok.getVec();

        } catch (SQLException ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        }
        return null;
    }
}
